# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
from pymongo import MongoClient
from scrapy.utils.project import get_project_settings

settings = get_project_settings()


class PkuDataPipeline:
    def __init__(self):
        self.mongo_client = MongoClient(
            host=settings["MONGO_HOST"],
            port=settings["MONGO_PORT"],
            username=settings["MONGO_USER"],
            password=settings["MONGO_PSW"],
        )
        self.mongo_db = self.mongo_client[settings["MONGO_DB"]]
        self.collect = self.mongo_db[settings["MONGO_COLLECT"]]

    def close_spider(self, spider):
        self.mongo_client.close()
        pass
    def process_item(self, item, spider):
        self.collect.insert_one(dict(item))
        return item
